/**
 *
 * Copyright Pact Lab of H.I.T.
 *
 * Designed and Implemented by Grid Researching Group, 
 * Pact Lab, Harbin
 * 
 * This Project is part of the national 973 Project:
 * Internet Based Virtual Computing Environment
 *
 * http://pact518.hit.edu.cn
 * 
 * Author:       Meteor <meteorlxk@gmail.com> 
 * Copyright:    pact518 
 * Version:      1.0
 * Created:      2009-5-4 
 * LastModified: 2009-5-4
 */
package edu.hit.pact.pgse.crawler.web;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.PriorityBlockingQueue;

import org.apache.commons.logging.LogFactory;

import edu.hit.pact.pgse.bean.RunnableTask;
import edu.hit.pact.pgse.crawler.Crawler;
import edu.hit.pact.pgse.crawler.CrawlerManager;
import edu.hit.pact.pgse.crawler.web.PageFetchProcessor;
import edu.hit.pact.pgse.crawler.finish.FinishedReportedProcessor;
import edu.hit.pact.pgse.crawler.send.RawPageSendProcessor;
import edu.hit.pact.pgse.crawler.util.DuplicateUrlFilter;
import edu.hit.pact.pgse.crawler.util.FileFilter;
import edu.hit.pact.pgse.crawler.util.RobotFileManager;
import edu.hit.pact.pgse.crawler.util.RobotFileScanner;
import edu.hit.pact.pgse.util.AbstractThread;
import edu.hit.pact.pgse.util.Globals;

/**
 * @author meteorlxk
 *
 */
public class WebCrawler extends Crawler{
	
	private PriorityBlockingQueue<UrlUnit> cleanUrlQueue;//use to keep the order of url, according to the number of backlinks
	private ConcurrentHashMap<String, UrlUnit> urlUnitMap;//use to trace the urlUnit and modify the backlinks
	
	private FileFilter fileFilter;
	private RobotFileManager robot;
	private DuplicateUrlFilter duplicateUrlFilter;
	
	private List<AbstractThread> threads;
	private CrawlerManager crawlerManager;
	
	
	public WebCrawler(CrawlerManager crawlerManager, RunnableTask runnableTask, FinishedReportedProcessor finishedReporter,  boolean ifSend, boolean ifStore) {
		super(runnableTask, finishedReporter, LogFactory.getLog(WebCrawler.class), ifSend, ifStore);
		
		this.crawlerManager = crawlerManager;
		this.cleanUrlQueue = pieceProperty.getCleanUrlQueue();
		this.urlUnitMap = pieceProperty.getUrlUnitMap();
		
		/*
		 * Construct FileFilter
		 */
		String supportedFileName = Globals.SUPPORTED_FILE_PATH;
		String unsupportedFileName = Globals.UNSUPPORTED_FILE_PATH;
		try {
			fileFilter = new FileFilter(supportedFileName, unsupportedFileName);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		this.duplicateUrlFilter = new DuplicateUrlFilter();
		
		RobotFileScanner scanner = new RobotFileScanner(fetcher);
		this.robot = new RobotFileManager(scanner);
	}
	
	public void start() throws InterruptedException {
		
		UrlUnit homePage = new UrlUnit(((WebTaskPiece)pieceProperty.getPiece()).getHomePage().toString(), 1);
		
		//in case the network environment is bad, try 3 times
		this.cleanUrlQueue.put(homePage);
		this.cleanUrlQueue.put(homePage);
		this.cleanUrlQueue.put(homePage);
		this.urlUnitMap.put(((WebTaskPiece)pieceProperty.getPiece()).getHomePage().toString(), homePage);

//		logger.error(pieceProperty.getCleanUrlQueueRemain()+"----"+(cleanUrlQueue.size()));
		logger.info("put home page into todoQueue");
		finishedReporter.start();
		
		if (ifSend){
			pageSendProcessor.start();
		}
		
		threads = new ArrayList<AbstractThread>();
		
		/*
		 * construct pagePrepareProcessor Thread
		 */
		PagePrepareProcessor pagePrepare = new PagePrepareProcessor(pieceProperty,fileFilter, robot, duplicateUrlFilter);
		
		/*
		 * construct pageFetchProcess Thread Group
		 */
		for (int i = 0; i < Globals.CRAWLER_PAGE_FETCH_THREAD_NUM; i++) {	
			PageFetchProcessor pageFetch = new PageFetchProcessor(crawlerManager, pieceProperty, fetcher, fetchedPageQueue, store, pagePrepare);
			threads.add(pageFetch);
		}//end for
		
		/*
		 * start all thread
		 */
		for (AbstractThread thread : threads) {
			thread.start();
		}//end for
	}
	
	public void deactivate(){
		logger.info("closing web crawler");
		super.deactivate();
		robot = null;
		
		for (AbstractThread thread : threads) {
			thread.setDeadLoop(false);
		}//end for
	}
}
